{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2-layer FNN on Cifar\n",
    "\n",
    "This is MLP (784-X^W-10) on MNIST. SGD algorithm (lr=0.1) with 100 epoches.\n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import numpy as np\n",
    "from matplotlib.pyplot import *\n",
    "import locale\n",
    "locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')\n",
    "\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.font_manager as font_manager\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\" Extract final stats from resman's diary file\"\"\"\n",
    "def extract_num(lines0):\n",
    "\n",
    "    valid_loss_str     = lines0[-5]\n",
    "    valid_accuracy_str = lines0[-6]\n",
    "    train_loss_str     = lines0[-8]\n",
    "    train_accuracy_str = lines0[-9]\n",
    "    run_time_str       = lines0[-10]\n",
    "\n",
    "    valid_loss     = float(valid_loss_str.split( )[-1])\n",
    "    valid_accuracy = float(valid_accuracy_str.split( )[-1])\n",
    "    train_loss     = float(train_loss_str.split( )[-1])\n",
    "    train_accuracy = float(train_accuracy_str.split( )[-1])\n",
    "    run_time       = float(run_time_str.split( )[-1])\n",
    "    \n",
    "    return valid_loss, valid_accuracy, train_loss, train_accuracy, run_time\n",
    "\n",
    "\"\"\" Extract number of total parameters for each net config from resman's diary file\"\"\"\n",
    "def parse_num_params(line0):\n",
    "    line_str = ''.join(lines0)\n",
    "    idx = line_str.find(\"Total params\")\n",
    "    param_str = line_str[idx+14:idx+14+20] # 14 is the length of string \"Total params: \"\n",
    "    param_num = param_str.split(\"\\n\")[0]\n",
    "    return int(locale.atof(param_num))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract results from diary file\n",
    "\n",
    "    1. Number of params\n",
    "    2. Loss/Accuarcy for training/testing\n",
    "    3. Runing time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../results/fnn_mnist_l2_dir/171003_210513_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_10/diary', '../results/fnn_mnist_l2_dir/171003_210328_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_0.001/diary', '../results/fnn_mnist_l2_dir/171003_210513_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_0.01/diary', '../results/fnn_mnist_l2_dir/171003_210342_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_10000/diary', '../results/fnn_mnist_l2_dir/171003_210532_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_100/diary', '../results/fnn_mnist_l2_dir/171003_210328_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_0.1/diary', '../results/fnn_mnist_l2_dir/171003_210328_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_0/diary', '../results/fnn_mnist_l2_dir/171003_210328_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_1/diary', '../results/fnn_mnist_l2_dir/171003_210520_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_0.0001/diary', '../results/fnn_mnist_l2_dir/171003_210335_9e4037a_lrb_fnn_mnist_sparse_dir_0_2_200_1000/diary']\n"
     ]
    }
   ],
   "source": [
    "l2_value = [10000,1000,100,10,1,0.1,0.01,0.001,0.0001,0] # [100,1,0.01,0]\n",
    "results_dir = '../results/fnn_cifar_l2_reg' # fnn_mnist_l2_dir, fnn_cifar_l2_0\n",
    "       \n",
    "depth = [2]\n",
    "width = [200]\n",
    "dim   = [0,250,500,750,1000,1250,1500,1750,1900,1950,2000,2050,2100,2250,2500,3000,4000,5000,5250,5500,5750,6000,6250,6500,6750,7000,7250,7500,7750,8000,8250,8500,8750,9000,9250,9500,9750,10000,15000,20000,25000,30000,35000,40000,45000,50000,100000,200000]\n",
    "\n",
    "########## 1. filename list of diary ########################\n",
    "diary_names = []\n",
    "for subdir, dirs, files in os.walk(results_dir):\n",
    "    for file in files:\n",
    "        if file == 'diary':\n",
    "            fname = os.path.join(subdir, file)\n",
    "            diary_names.append(fname)\n",
    "            \n",
    "print diary_names\n",
    "            \n",
    "########## 2. Construct stats (width, depth, dim) ##########\n",
    "# acc_test_all : Tensor (width, depth, dim)\n",
    "# num_param_all: Tensor (width, depth)\n",
    "############################################################\n",
    "nw, nd, nn, nl= len(width), len(depth), len(dim), len(l2_value)\n",
    "\n",
    "acc_test_all  = np.zeros((len(width), len(depth), len(l2_value)))\n",
    "num_param_all = np.zeros((len(width), len(depth)))\n",
    "\n",
    "mode = 1        # {0: test loss, 1: test acc}\n",
    "error_files = [] #  record the error file\n",
    "dim = []\n",
    "\n",
    "# 2.1 construct acc_test_all and num_param_all\n",
    "for i in range(len(l2_value)):\n",
    "    d = l2_value[i]\n",
    "\n",
    "    # 2.1.1 Read the results, \n",
    "    for f in diary_names:\n",
    "        if '_'+str(d)+'/' in f:\n",
    "            # print \"%d is in\" % d + f\n",
    "\n",
    "            str_tmp = f.split('_')\n",
    "            str_tmp[-1]\n",
    "\n",
    "            with open(f,'r') as ff:\n",
    "                lines0 = ff.readlines()\n",
    "                try:\n",
    "                    R = extract_num(lines0)\n",
    "                    R = np.array(R)\n",
    "\n",
    "                except ValueError:\n",
    "                    error_files.append((w,ll,d))\n",
    "                    R = np.zeros(len(R))\n",
    "                    print \"Error. Can not read config: depth %d, width %d and dim %d.\" % (ll, w, d) \n",
    "                    # break\n",
    "\n",
    "\n",
    "    # 2.1.2 Assign the results\n",
    "    r = R[mode]  \n",
    "    acc_test_all[0,0,i]=r\n",
    "\n",
    "########## 3. Construct Tensors for Analysis (width, depth, dim) ##########                    \n",
    "acc_base  = acc_test_all[:,:,0]\n",
    "\n",
    "                                       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline results\n",
      "[ 0.9839  0.9836  0.9829  0.9831  0.9835  0.9829  0.9843  0.9839  0.9848\n",
      "  0.9836]\n"
     ]
    }
   ],
   "source": [
    "print \"Baseline results\"\n",
    "print acc_test_all[0,0,:] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing Accuracy wrt. Width, Depth and Dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.font_manager as font_manager\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize=(20,5.0))\n",
    "\n",
    "for i in range(acc_test_all.shape[2]):\n",
    "    acc = acc_test_all[:,:,i].reshape(len(depth)*len(width),order='F')\n",
    "    if i==0:\n",
    "        plt.scatter(fig_params_1d, acc, s=(np.array(fig_width)**1.8)/100, c=fig_depth, edgecolors='k') \n",
    "        plt.scatter(fig_params_1d, sr*acc, marker=\"_\", s=300, c='k', edgecolors='r') \n",
    "    else:\n",
    "        plt.scatter(fig_params_1d, acc, s=(np.array(fig_width)**1.8)/100, c=fig_depth, facecolors='None', linewidth=np.array(dim[i])/300.0) \n",
    "\n",
    "        \n",
    "ax = plt.gca()\n",
    "plt.colorbar(label=\"Depth\")\n",
    "\n",
    "ax.set_xscale('log')\n",
    "ax.grid(True)\n",
    "\n",
    "ax.set_ylim(0.1, 1.0)\n",
    "ax.set_xlim(0.3E5, 1.5E6)\n",
    "\n",
    "plt.xlabel('# parameters')\n",
    "plt.ylabel('# accuracy')\n",
    "\n",
    "#make a legend:\n",
    "pws = width\n",
    "for pw in pws:\n",
    "    plt.scatter([], [], s=(pw**1.8)/1000, c=\"k\",label=str(pw))\n",
    "\n",
    "h, l = plt.gca().get_legend_handles_labels()\n",
    "plt.legend(h[0:], l[0:], labelspacing=1.2, title=\"layer width\", borderpad=1, loc='best', bbox_to_anchor=(1.25, 1),\n",
    "             frameon=True, framealpha=0.6, edgecolor=\"k\", facecolor=\"w\")\n",
    "\n",
    " \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Intrinsic dim for #parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'fig_params_1d' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-0894e4317b48>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig_params_1d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim_solved_all_1d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig_width\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfig_depth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0medgecolors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'k'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgca\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'fig_params_1d' is not defined"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9067c9aa10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(20,5))\n",
    "\n",
    "plt.scatter(fig_params_1d, dim_solved_all_1d, s=(np.array(fig_width)**2.0)/100, c=fig_depth, edgecolors='k') \n",
    "\n",
    "ax = plt.gca()\n",
    "plt.colorbar(label=\"Depth\")\n",
    "\n",
    "ax.set_xscale('log')\n",
    "\n",
    "ax.grid(True)\n",
    "ax.set_ylim(0, 700)\n",
    "\n",
    "plt.xlabel('# parameters')\n",
    "plt.ylabel('# dimension')\n",
    "ax.set_xlim(0.3E5, 1.5E6)\n",
    "\n",
    "#make a legend:\n",
    "pws = width\n",
    "for pw in pws:\n",
    "    plt.scatter([], [], s=(pw**1.8)/100, c=\"k\",label=str(pw))\n",
    "\n",
    "h, l = plt.gca().get_legend_handles_labels()\n",
    "plt.legend(h[0:], l[0:], labelspacing=1.2, title=\"layer width\", borderpad=1, \n",
    "            frameon=True, framealpha=0.6, edgecolor=\"k\", facecolor=\"w\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance comparison with Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f90658fe890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEUCAYAAADOaUa5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXFWZ//HPl7AGURKBoBJIEAUCbpBxdEToMGoQFaI4\n7AgRwuKIoOIL+AUYhERRFJS4gSA6LMEBWRUBgTSy5aeAggkQARMIDLKYsHRCQoLP/HFuQeVS1X2r\nu7q76vb3/XrVK6lTz7n3OV1dT90+detcRQRmZlZOqw12AmZm1n9c5M3MSsxF3sysxFzkzcxKzEXe\nzKzEXOTNzErMRd7MrMRc5M3MSsxF3sysxFYf7ARa0QYbbBBjxoxpqM+SJUtYd911+yehFuexD82x\nw9Aef2/Gfvfddz8bERv2U0o1ucjXMGbMGO66666G+nR2dtLR0dE/CbU4j71jsNMYNEN5/L0Zu6RH\n+yeb+jxdY2ZWYi7yZmYl5iJvZlZiLvJmZiXmIm9mVmIu8mZmJeYib2ZWYi7yZmYl5iJvZlZiLvJm\nZiXmIm9mVmIDXuQlbSJphqQ7JS2VFJLGFOy7mqTjJS2QtEzSvZL2qBM7RdKDkpZLmifp8GaOw8ys\nHQzGkfwWwJ7AYuDWBvueCpwM/AD4ODAbuFTSrtVBkqYAZwO/AnYBLgV+JOmIPmVuZtZmBmMVyt9H\nxCgASYcAHyvSSdJGwDHAaRHxnax5lqQtgNOAa7O41YHpwAURMbUq7q3AqZLOjYgVzRuOmVnrGvAj\n+Yj4Zy+7TgTWBC7MtV8IvEvS2Oz+B4ENa8RdALwZ2KGX+zczazvt9MHrNsBy4OFc+9zs33FVcQBz\neogzMyu9drpoyEjguYiIXPuiqser/13cQ9wqJB0KHAowatQoOjs7G0quq6ur4T5l4bF3DnYag2Yo\nj79dxt5ORb5fRcQ5wDkA48ePj0av+OIr5HQMdhqDYiiPHYb2+Ntl7O00XbMYWF+Scu2VI/NFVXEA\nI3qIMzMrvXYq8nOBtYC359orc+z3V8XBa3Pz9eLMzEqvnYr8dcAKYL9c+/7AnIiYn92/E3i2Ttwi\n4Pb+TNLMrJUMypy8pM9m/90++/fjkp4BnomIW7KYlcAvIuJggIh4WtIZwPGSXgTuAfYCdgZ2q2w7\nIlZIOpH05acngBuzmM8DR0bEy/0/QjOz1jBYH7xemrv/o+zfW4CO7P/Dslu1qUAXcBSwMTAP2DMi\nfl0dFBE/kRTAV4GvAY8BX4yIH2FmNoQMSpGPiPyHp4ViIuIVYFp266n/2aSlDczMhqx2mpM3M7MG\nucibmZWYi7yZWYm5yJuZlZiLvJlZibnIm5mVmIu8mVmJucibmZWYi7yZWYm5yJuZlZiLvJlZibnI\nm5mVmIu8mVmJucibmZWYi7yZWYm5yJuZlZiLvJlZibnIm5mVmIu8mVmJucibmZWYi7yZWYm5yJuZ\nlZiLvJlZibnIm5mVmIu8mVmJucibmZWYi7yZWYm5yJuZlZiLvJlZibnIm5mVmIu8mVmJucibmZXY\ngBd5SaMlXSbpeUkvSLpc0qYF+p0sKercluViF9SJm9R/IzMzaz2rD+TOJA0HbgaWAwcCAUwDZkl6\nd0Qs6ab7ucB1ubZ1s7ara8RfD5yca5vXi7TNzNrWgBZ5YAqwObBlRDwMIOk+4CHgMOCMeh0j4nHg\n8eo2SQeQxvCLGl2ejYjZTcrbzKwtDfR0zW7A7EqBB4iI+cDtwO692N6BwFOko3YzM8sZ6CK/DTCn\nRvtcYFwjG5I0GpgAXBQRK2uEfErSUknLJc32fLyZDUUDXeRHAotrtC8CRjS4rf1J+deaqrkGOBKY\nCOwHLAOukLR/g/swM2trioiB25n0MnBGRByXa58GHBcRhT8jkPQA8FJEbFcgdhgwG9g4IkbXiTkU\nOBRg1KhR219yySVFUwGgq6uLN7zhDQ31KQuPfWiOHYb2+Hsz9gkTJtwdEeP7KaWaBvqD18XUPmKv\nd4Rfk6T3A1sBRxeJj4hXJF0KfEvSWyLiyRox5wDnAIwfPz46OjqKpgNAZ2cnjfYpC4+9Y7DTGDRD\nefztMvaBnq6ZS5qXzxsH3N/Adg4EVgAX9yKHgfvTxcxskA10kb8a+ICkzSsNksYAH6L2ue6vI2lN\nYG/gtxHxTME+qwN7AY9FxN8bzNnMrG0NdJH/KbAAuErS7pJ2A64CFgJnV4IkbSZppaSTamzjk6Tp\nnVofuCJpH0mXSPqcpAmS9gZmAdsBxzZ3OGZmrW1A5+QjYomknYEzgQsAATcBR0dEV1WogGHUfhM6\nkHQ2zq/r7GY+sBFwOunNYAlwF7BLRPh8ejMbUgb6g1ci4jFgjx5iFpAKfa3Huv3SVPYt1517m5+Z\nWZl4FUozsxJzkTczKzEXeTOzEitU5CXVnB83M7PWVvRI/lFJJ0p6a79mY2ZmTVW0yN8MHAcsyK7k\n9LF+zMnMzJqkUJGPiIOAtwLHAO8ErpP0iKRjJW3Yj/mZmVkfFP7gNSKej4izImJbYCfgDtLl9RZm\n3zDt6J8Uzcyst3p7ds3twBXAn4E1gU8BN0n6g6Stm5WcmZn1TUNFXtJoSacAjwH/AzxHumzfesAu\nwDrUWVPGzMwGXqFlDSR9inSh7YnA88D5wI8j4m9VYb+T9BXgN03P0szMeqXo2jVXAX8EDgEuiYjl\ndeIeAS5qRmJmZtZ3RYv8+Ii4p6eg7Mh+ct9SMjOzZik6J79Q0jtrPSDpnZI2aGJOZmbWJEWL/I+A\nr9Z57MvZ42Zm1mKKFvkdgHoX3LiBdPk+MzNrMUWL/AjSWTW1vAC8uTnpmJlZMxUt8o8D/1rnsX8F\nnmxOOmZm1kxFi/xlwPGSPlHdmN0/jvTFKDMzazFFT6E8BdgRuFrS34EngLcBGwOzga/3T3pmZtYX\nhYp8RCyVtBNwAPBR0hz8w6QPXS+MiJX9l6KZmfVW0SN5ImIF8LPsZmZmbcDXeDUzK7HCR/LZ1aCO\nALYE1s49HBHx9mYmZmZmfVf0Qt67Ar8FhgNbAQ+SlhseDfwT+H1/JWhmZr1XdLrmROCHwK7Z/RMi\nogPYBhhGegMwM7MWU7TIbwVcQzpqD7Jpnoj4K+kSgCf2R3JmZtY3RYv8P4GVERHAM8CmVY/9L+D5\neDOzFlS0yM8DxmT/vws4WtJbJG1IWp1yQfNTMzOzvip6ds1FQOUC3f8F3EhazwbgFWDfJudlZmZN\nUPQbrz+s+v/dkt5FunD3cODGiLi/n/IzM7M+6LHIS1qTdH78TRExByAiHgfO7efczMysj3qck4+I\nl4HTgJHN2KGk0ZIuk/S8pBckXS5p0557gqSoc3tvLm41ScdLWiBpmaR7Je3RjPzNzNpJ0Q9eHwA2\n7+vOJA0HbiadknkgacGzdwCzJK1bcDM/Bz6Yu/01F3Mq6dTOHwAfJ62UeWn2pS4zsyGj6AevJwHf\nl3R3RPylD/ubQnqz2DIiHgaQdB/wEHAYcEaBbTwREbPrPShpI+AY4LSI+E7WPEvSFqS/SK7tQ/5m\nZm2laJE/FngD8CdJC0hXgoqqxyMidiqwnd2A2ZUCn3WcL+l2YHeKFfmeTATWBC7MtV8I/EzS2IiY\n34T9mJm1vKLTNa8A9wO3AguBlVlb5fbPgtvZBphTo30uMK7gNo6QtFzSUkk3S/pwjX0sJ613n98H\nDezHzKztFT2FsqNJ+xsJLK7Rvoh0sfCeXAj8mvQt282ArwE3S/poRHRW7eO57Nu5+X1UHjczGxIK\nLzXcCiLigKq7t0q6ivSXwTRgh75sW9KhwKEAo0aNorOzs6H+XV1dDfcpC4+9c7DTGDRDefztMvZC\nRV7Sjj3FRESR5YYXU/uIvd4Rfk/7fFHSb4CDc/tYX5JyR/OVI/hF1BAR5wDnAIwfPz46OjoayqWz\ns5NG+5SFx94x2GkMmqE8/nYZe9Ej+U5W/aC1lmEFtjOXNGeeN440599b1bnNBdYiLZpWPS9fmYv3\nt3PNbMgoWuQn1Gh7M/BJYCfgiwW3czXwHUmbR8TfACSNAT4EHFdwG6+S9MYshz9UNV8HrAD2A75e\n1b4/MMdn1pjZUFL0g9db6jx0uaQzgU9R7MIhPyW9IVwl6QTSEfippDN2zq4ESdoMeAQ4JSJOydqO\nIV16cBavffB6DLAxqaBXcn1a0hnA8ZJeBO4B9gJ2Jp3CaWY2ZDTjg9ffAJcAX+gpMCKWSNoZOBO4\nABBwE3B0RHRVhYo0/VN9iuc84NPZ7U3AC8DtwMERUX0kDzAV6AKOIr0JzAP2jIhfNzw6M7M21owi\nvyXFz5MnIh4Dul1HJiIWkAp9dds1pKtTFdnHK6QzbqYVzcvMrIyKnl3zuRrNawLbks5subyZSZmZ\nWXMUPZL/eZ325cAvSdMiZmbWYooW+bE12pZFxFPNTMbMzJqr6Nk1j/Z3ImZm1nyFFiiT9ElJNc+F\nl/SfXqfdzKw1FV2F8kSg3kU91skeNzOzFlO0yG9F+lJRLX8Gtm5OOmZm1kxFi/xqpIuG1LIesEZz\n0jEzs2YqWuTvpWrpgJz9gPuak46ZmTVT0VMovwv8StKlpPVnHgfeRlp//dPAf/RPemZm1hdFT6G8\nQtJRwHTgM1mzSOvDfCki/I1XM7MWVHjtmoiYIennwL+Rlhl+Frgjt7CYmZm1kIYWKIuIF4Hr+ykX\nMzNrsqJfhjpW0ow6j50l6WvNTcvMzJqh6Nk1k6l/Bs2fs8fNzKzFFC3ymwIP1Xnsb6SrNJmZWYsp\nWuSXkk6ZrGUT0pLDZmbWYooW+VuBr0laq7oxu//V7HEzM2sxRc+uORm4A/irpAuBJ0hH9vuTTqc8\nqD+SMzOzvin6Zah7JU0AvgMcS/oL4J/AbcAeEXFv/6VoZma9VXS6hoj4Q0TsSFqQbBNgvYjoANaV\n9LN+ys/MzPqgcJGviIiXgOHA8ZLmA7OAPZudmJmZ9V3hIi/pTZIOlXQ7MA+YCiwGjgDe2k/5mZlZ\nH3Rb5CWtJmlXSb8EngR+Qjon/odZyNERcXZEvNDPeZqZWS/U/eBV0neBfYGNgGXAFcAvgBuBNwI1\nr/lqZmato7uza74MBHAtcFBE/KPygKTo78TMzKzvupuuOQ94EfgEME/SDyS9f2DSMjOzZqhb5CNi\nCrAx6fJ+dwGHAXdKeoB0rryP5s3MWly3H7xGxLKImBkRu5AWKTseeAU4jnRlqNMk7S9p7f5P1czM\nGtXIl6GejIhvR8S2wPtJZ9i8A/hv0pk3ZmbWYhr+MhRARNwVEUeSzo/fA+hsZlJmZtYcDV3+Ly8i\nVpBOrbyiOemYmVkz9epI3szM2sOAF3lJoyVdJul5SS9IulzSpgX6jZd0jqQHJS2V9JikiySNrRG7\nQFLUuE3qn1GZmbWmPk3XNErScOBm0pWkDiSdhjkNmCXp3RGxpJvuewPbAGcBc0nr2Z8I3CXpvRGx\nMBd/PWkd/Grz+jwIM7M2MqBFHpgCbA5sGREPA0i6j3T92MOAM7rp+62IeKa6IVssbX623ZNy8c9G\nxOxmJW5m1o4GerpmN2B2pcADRMR84HZg9+465gt81vYo8Az1rz9rZjakDXSR3waYU6N9LjCu0Y1J\n2pq0gNoDNR7+VDZ3v1zSbM/Hm9lQNNDTNSNJa9DnLQJGNLIhSauTlj5+hrTOTrVrgD+SpnJGkVbM\nvELSARFxYZ3tHQocCjBq1Cg6OzsbSYeurq6G+5SFx9452GkMmqE8/rYZe0QM2A14GTitRvs0YGWD\n2/oJsAL4WIHYYaSiv7DItrfffvto1KxZsxruUxYe+9A1lMffm7EDd8UA1tyIGPDpmsXUPmKvd4Rf\nk6TTSEfdn4+IG3qKj4hXgEuBTSS9peh+zMza3UBP18wlzcvnjQPuL7IBSVNJq2AeGREX9CIHr55p\nZkPGQB/JXw18QNLmlQZJY4APZY91S9KXSFM7UyPiB0V3ms3f7wU8FhF/bzBnM7O2NdBF/qfAAuAq\nSbtL2g24ClgInF0JkrSZpJWSTqpq2xv4HnAdcLOkD1TdxlXF7SPpEkmfkzQh6zcL2I70F4CZ2ZAx\noNM1EbFE0s7AmcAFpDXpbyJdELyrKlSkD0ur34R2ydp3yW7VbgE6sv/PJ51WeTpprn8J6aInu0TE\n9c0cj5lZqxvoOXki4jHS8sTdxSwgFfTqtoOAgwpsfzawc68TNDMrEa9CaWZWYi7yZmYl5iJvZlZi\nLvJmZiXmIm9mVmIu8mZmJeYib2ZWYi7yZmYl5iJvZlZiLvJmZiXmIm9mVmIu8mZmJeYib2ZWYi7y\nZmYl5iLfRzNnzmTbbbfl7rvvZtttt2XmzJmDnZKZ2atc5Ptg5syZTJ48mYULFwKwcOFCJk+ezOjR\no13szawluMj3wUEHHcSIESO48sor2W677bjyyisZMWIETz31FFOnTnWhNyuhdvvr3UW+D9Zee20u\nvvhiJkyYgCQmTJjAxRdfzDrrrMN5553H9OnTBztFM2uimTNnMnXqVGbMmMF2223HjBkzWv6AzkW+\nD1588UV22GGHVdp22GGHV9sfeOCBQcrMzPrD9OnTOe+881Y5sGv1AzoX+T5Yb731uO2221Zpu+22\n215t33rrrQcpMzPrDw888EDNA7tWPqBzke+DpUuXsu+++zJr1iwiglmzZrHvvvuydOlSDj74YKZO\nnTrYKZpZE2299dY1D+xa+YDORb4PDj/8cBYvXsykSZO45557mDRpEosXL+aNb3wj06dPZ5999hns\nFM2siaZOncrBBx+8yoFdqx/QrT7YCbSzGTNmAHDRRRcBMGzYMKZMmfJqu5mVS+XA7cgjj2Ty5Mmc\nf/75LX9A5yP5PpoxYwaLFi1i++23Z9GiRS7wZiW3zz77MGfOHLbffnvmzJnT0gUeXOTNzErNRd7M\nrMRc5M3MSsxF3sysxFzkzcxKzEXezKzEXOTNzErMRd7MrMQGvMhLGi3pMknPS3pB0uWSNi3Yd21J\np0t6UtJLku6UtGONuNUkHS9pgaRlku6VtEfzR2Nm1toGtMhLGg7cDGwFHAgcALwDmCVp3QKbOA+Y\nApwEfBJ4Erhe0ntzcacCJwM/AD4OzAYulbRrE4ZhZtY2BnrtminA5sCWEfEwgKT7gIeAw4Az6nWU\n9B5gX+DzEXF+1nYLMBc4Bdgta9sIOAY4LSK+k3WfJWkL4DTg2n4Yl5lZSxro6ZrdgNmVAg8QEfOB\n24HdC/RdAfyyqu9K4BJgoqS1suaJwJrAhbn+FwLvkjS2TyMwM2sjA13ktwHm1GifC4wr0Hd+RCyt\n0XdNYIuquOXAwzXiKLAfM7PSGOgiPxJYXKN9ETCiD30rj1f+fS4iooc4M7PS83ryGUmHAocCjBo1\nis7Ozob6d3V1NdynLDz2zsFOY9AM5fG3y9gHusgvpvYRe72j9Hzfzer0hdeO1BcD60tS7mg+H7eK\niDgHOAdg/Pjx0dHR0UM6q+rs7KTRPmXhsXcMdhqDZiiPv13GPtDTNXNJc+Z544D7C/Qdm52Gme/7\nMq/Nwc8F1gLeXiOOAvsxMyuNgS7yVwMfkLR5pUHSGOBD2WPduQZYA/iPqr6rA3sBN0TE8qz5OtJZ\nOPvl+u8PzMnO5jEzGxIGerrmp8AXgasknQAE6YtLC4GzK0GSNgMeAU6JiFMAIuJPkn4JfE/SGsB8\n4AhgLFUFPSKelnQGcLykF4F7SG8EO5OdS29mNlQMaJGPiCWSdgbOBC4ABNwEHB0RXVWhAobx+r80\nJgPTgWnA+sC9wC4RcU8ubirQBRwFbAzMA/aMiF83d0RmZq1twM+uiYjHgG7XkYmIBaRCn29/CfhK\nduuu/yukN4JpvU7UzKwEvAqlmVmJucibmZWYi7yZWYm5yJuZlZiLvJlZibnIm5mVmIu8mVmJucib\nmZWYi7yZWYm5yJuZlZiLvJlZien1V8kzSc8AjzbYbQPg2X5Ipx147EPXUB5/b8a+WURs2B/J1OMi\n3ySS7oqI8YOdx2Dw2Ifm2GFoj79dxu7pGjOzEnORNzMrMRf55jlnsBMYRB770DWUx98WY/ecvJlZ\niflI3sysxFzke0nSaEmXSXpe0guSLpe0aQvk9VlJv5L0qKSXJM2T9E1J61XFjJEUdW7r57a3tqTT\nJT2Zbe9OSTvW2O9qko6XtEDSMkn3Sqp5mUdJUyQ9KGl5lt/hdeImSfpTtr1HJZ0gaVg3Y++oM6bn\ncnEjJJ0r6VlJSyTdKOldNbbXNmPP+nR287xel8W0/XMvaRNJM7J8lma5j6mxjZbMPxe3g6Q7svz+\nLukMSevUiNtG0g2SuiT9Q9L5kkbW2vfrRIRvDd6A4cBDwBxgErA78BfgEWDdQc5tNvA/wH7ATsDR\nwHNZ+2pZzBgggG8AH8jdhuW2d1HWfwrw78DlwEvAe3Nx04HlwDHABOBs4J/Arrm4KVn79CxuWnb/\niFzcROAV0rznBNJ1fZcB3+pm7B3ZuI7MjWl8VYyA24DHgX2AXYBbSOc7b9KuY8/6javxfH45+5l8\noSzPffY8PwVcC1yfjWdMjZ9HS+ZfFffuLJ8rs/wOARYDv8zFvRV4BriV9Pu6N7AQuIPsNd3t78Vg\nFqR2vQFHZU/iFlVtY4GVwFcGObcNa7R9Lnsh7Jzdr7zQD+lhW+/J4iZXta0OzAOurmrbKHuRfD3X\n/ybgvlzfp4Ff5OJ+Riqya1S1/Qm4JRd3EvAysHGdfDuyfD/SzZh2z2ImVLW9CVgEnNWuY+9mvOdl\n+Y0sy3NPVWEjFcbXFflWzr+q7QrSwWJ138prdbuqtjNJb1brV7XtmMV9psffgUZ+YXxb5Rfg9hrt\nt+Sf3Fa4AVtnvxAHZPeLvtBPzH4xh+fav569MNbK7h+Qbe8dubjJWfvY7P6Hs/sfzcVNoKrwAqOz\n+1NycWPzL9zc4x30XOTPA56o0f4L4NF2HXudsQ4HXgAuq2or1XNP/SLf0vkDa5CO4qfn4tYm96ZD\nmiG4qMZz9Ci5N51aN8/J9842pKmavLmkP5lbzU7Zvw/k2r8paaXS5wpX15iX3gaYHxFLc+1zgTWB\nLarilgMP14iD134m22T/5n92heIiYj6wlJ5/xhdJeiWbu7xYq35W0t1zt6mkN1TFtePYq30aWI/0\nBpZX1ue+XfJ/O6mg5+OWkYr6OIBsfn5sjf1W9t3jz2P1ngKsppGkubO8RcCIAc6lW5LeBpwC3BgR\nd2XNy0nzjjeQ5vq2Av4fcIek90dE5c2gu3FWHq/8+1xkhxc9xFFjm0XjKm31PnB6Hvgu6S+qF4D3\nkcZ1p6T3RcTTWd8FNfpWchgBdNF+Y6/lc6Qpht9WtZX1uc9r9fy7i1tU9fgI0udI9eK2rNG+Chf5\nEsuOSq8ifVYwudIeEU8C1WcF3JqdfTEXmArsP5B5NktE/Ik0H1pxi6TfA38AvgScMCiJDQJJbwU+\nAnw/IlZW2sv63Ft9nq7pncXUPmKvd/Qw4LI/864BNgcmRsTj3cVHxELSWSf/UtXc3TjhtaOYxcD6\nklQgjhrbLBpXaVtUo72miLgH+CuvjaunMS0uGNfqY9+f9PquNVWzipI+962ef3dxI6viniPN5fcU\nV5eLfO/M5bW5t2rjgPsHOJfXkbQGcBkwnnQa2F8a6F79Z+tcYKyk4bmYcaQPtR6uiluLNM+Yj4PX\nfiaV+cv8z65QXHYu9HB69zOujKu75+6xiOiqimvnsR8I3BsR9xaMh3I9962e/yOkqbN83NqkA7P7\nAbLPFBbU2G9l3z3/PIp+Uu/bKp9qH02aAtm8qm0MsAL46iDnthrpPPmXgH9voN+mpHns/65qex/p\nhX9gVdvqpA9wr6lq24j0wvmv3DZvBP5SdX8N0jzw+bm4c4F/AGtWtf0ZmJWLO4EGTyMkvdG9ApyS\n3Z+UjWmnqpg3ZvufUYaxZ2MO4Mtlf+6pf3ZNy+dPOj/+r8DqVW37Z3lvX9X2PdKR/5uq2nbI4vbo\n8fkt+mLxbZUnbF3SkcBfSOdd7wbcC/wNeMMg5/bj7Mmfxuu/7LJJFvNd0rm3e5JOATucdDrWc8CW\nue1dkv2CHUL6wsZlpC92bJeLOy1r/wrpVMYfk74o8slc3OFZ+7Qs7pTs/n/m4nbN2s/O4r6cbf/0\nbsZ+UbbdzwA7A18lncP8GLBBFrMa6UskC0lfKpkIdJL+7B3drmPP9T+LdMCxUY3HSvHcA5/NbpXf\n9yOy+zu1Q/5Z3Huz9suz/A4m/R5emot7G+n3+BbSl6H2yp6zV7/g2O3vw2AWpHa+kY5+fkU6AnqR\n9K48pgXyWpD90te6nZzFfB74Y/YCWAH8Hbg4/yLPYtcBzshilgH/H+ioETeMdLTyKOnP0PuAz9bJ\n8TDSEcxy0pdBvlAn7jOkN8/lpEJ9ErlvZebij8/2+3w2roWkbx2+JRc3kvQllkWk09puAt7TzmOv\n6lc54rymzuOleO67+R3vbIf8q+J2BO7M8nuKdNQ+vEbcu4DfAUuy5+7nwJuL1ASvQmlmVmL+4NXM\nrMRc5M3MSsxF3sysxFzkzcxKzEXezKzEXOTNzErMRd4GhaSDssu2bdFz9Ov6TpL0lQb7nCypX84X\nbua2terl+FZIekbSrZJOlLRRjfhOSZ3N2LeVk4u8taNJpG8nNuJc4IP9kEt/bPvn2fZ2In156fek\nSxrOlfRvudgvZDezmrzUsJWapLUiYnmkVTi7XYmzt/ph209ExOyq+9dIOot0jc/LJW0e2cUwImLQ\nF8Sz1uYjeWsZ2dTDbZI+IukeSUslzZH06aqYn5NWWHxb1bTGguyxjuz+ZyT9VNIzpK+K15xSyWKn\nSfqSpPmSXpR0i6T8yoATJd2RXUWpS9I8SSdVPV5r26tLOlbS/ZKWZdMu10naqjc/m4h4CvgaMIp0\nAfLqn1ln1f3Kz2CSpLMlLZL0nKTvSRom6V+yn/ESSXMlTexNPtY+fCRvrebtwPeBb5IWZfoqcKmk\nrSLiYeBUYEPS2ue7ZX2W57Yxg3Q1pANIl1jrzv6kizsfRbos3OnAVdn+VkraHLiatLjVKaSVBN9B\nWg62O5eQppW+R1rRcG3SOiVvAR7soW89N5BWP/0Q6Vq13fkeaeGrvbL9nkBao+UjpDE+kbVdLmmz\niHi2lzkREykYAAACRUlEQVRZi3ORt1azAbBjRDwEIOke4EnSqonfiIhHsiP0l3NTGtX+EBGHFNzf\nCtJqgyuy/QFcCryftFrldqTif0REvJD1ubm7DUraGdgDOCoizqp66MqCOdUUES9Jepb0RtGTmyOi\n8rnF7yR9Avgi8OGIuC3L80nSIlqfoMDFRaw9ebrGWs1DlQIPEOm6rE+TVv0s6ooGYn9XKfCZygVW\nKvv7M+mN4BJJn611hksNHyOtiPjTBvIoSqx6cY96fpu7/yCwpFLgq9oARjcjMWtNLvLWampdzmw5\nPU+7VHuyD/urTP2sDZBNEU0kvVYuAP4uabaknbrZ5puBRRHxUgN59Ci7pOMGFBtf/jKUL5PWjH9V\nRLyc/beRn621GRd5K6Omng8fEbMiYhdgfdKc9krgN5I2qNPlWWBkVpSbaSJpXv22ngLNKlzkrR0t\nJ10QYkBlp2LeDHybdHWwsXVCbyBNqxT9XKBH2TTRt0lH8Zc0a7tWfv7g1drR/aQj5SOAu4Bl0djF\nyguTdDjp7JRrSVea2oB0Bar/BebU6hMRsyT9CjhD0mjSB7VrZNv5TUR09rDbt0n6AOkgbCTp0o1T\nSG8cn2r2NJCVm4u8taNzSYXvG6QplEdJF1LvD/cCHyed0rkRaQ7/NmC/Hort3sCxpHP6jyZdkvCP\nWe49OSi7rcz6PUg6LfTsiHimN4OwocuX/zMzKzHPyZuZlZiLvJlZibnIm5mVmIu8mVmJucibmZWY\ni7yZWYm5yJuZlZiLvJlZibnIm5mV2P8BnsCxyN9N9I4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9065b1c810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(5,4))\n",
    "fig.subplots_adjust(hspace=0.3)\n",
    "fig, ax = subplots(figsize=(5,4) )\n",
    "\n",
    "font = {'family' : 'normal',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 16}\n",
    "matplotlib.rc('font', **font)\n",
    "\n",
    "for i in range(nw):\n",
    "    for j in range(nd):\n",
    "        id = i*nd+j+1\n",
    "        ax = plt.subplot(nw, nd, id)\n",
    "\n",
    "        plt.scatter(l2_value, acc_test_all[i,j,:], edgecolor=\"k\", facecolor=\"w\" )\n",
    "        ax.set_xlabel('Intrinsic Dim')\n",
    "        ax.set_ylabel('Accuracy')\n",
    "        # ax.set_title('width %d, depth %d' %(width[i], depth[j]))\n",
    "        plt.grid()\n",
    "        ax.set_ylim([-0.1,1.1])\n",
    "        # ax.set_xscale('log')\n",
    "        \n",
    "#fig.savefig(\"fnn_cifar.pdf\", bbox_inches='tight')        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
